# -*- coding: utf-8 -*-
import scrapy
from scrapy import FormRequest, Request
import re
from weibosearch.items import WeiboItem


class WeiboSpider(scrapy.Spider):
    name = 'weibo'
    allowed_domains = ['weibo.cn']
    # 'https://weibo.cn/search/mblog?hideSearchFrame=&keyword=000001'，两个参数一个是有的，一个是没有的（可忽略）
    search_url = 'https://weibo.cn/search/mblog'
    # 经过测试最多获取200页的内容
    max_page = 100

    def start_requests(self):
        keyword = '000001'
        url = '{url}?keyword={keyword}'.format(url=self.search_url, keyword=keyword)
        # for page in range(self.max_page + 1):
        for page in range(1,3):
            data={
                'mp':str(self.max_page),
                'page':str(page)
            }
            yield FormRequest(url, callback=self.parse_index, formdata=data)

    def parse_index(self, response):
        weibo_list = response.xpath('//div[@class="c" and contains(@id, "M_")]')
        for weibo in weibo_list:
            is_forward = bool(weibo.xpath('.//span[@class="cmt"]').extract_first())
            print(is_forward)
            if is_forward:
                detail_url = weibo.xpath('.//a[contains(., "原文评论[")]/@href').extract_first() #第一个参数是点，代表选择当前的文本
            else:
                detail_url = weibo.xpath('.//a[contains(., "评论[")]/@href').extract_first() #第一个参数是点，代表选择当前的文本
            print(detail_url)
            yield Request(detail_url, callback=self.parse_detail)

    def parse_detail(self, response):
        url = response.url
        id = re.search('comment\/(.*?)\?', url).group(1)
        content = ''.join(response.xpath('//div[@id="M_"]//span[@class="ctt"]//text()').extract())
        print(id, url, content)
        # 中括号前必须加转义字符 \
        comment_count = response.xpath('//span[@class="pms"]/text()').re_first('评论\[(\d*?)\]') #“\d”换成“.”效果一样的
        forward_count = response.xpath('//a[contains(., "转发[")]/text()').re_first('转发\[(\d*?)\]')
        like_count = response.xpath('//a[contains(., "赞[")]/text()').re_first('赞\[(.*?)\]')
        # 数据清理（也可放在pipeline中）
        comment_count = comment_count if comment_count is not None else 0
        forward_count = forward_count if forward_count is not None else 0
        like_count = like_count if like_count is not None else 0
        print(comment_count, forward_count, like_count)
        posted_at = response.xpath('//span[@class="ct"]/text()').extract_first()
        user = response.xpath('//div[@id="M_"]/div[1]/a/text()').extract_first(default=None) #default=None加不加无所谓
        print(posted_at, user)
        weibo_item = WeiboItem()
        for field in weibo_item.fields:
            try:
                # eval() 函数用来执行一个字符串表达式，并返回表达式的值
                weibo_item[field] = eval(field)
            # items.py中定义的没有爬取，捕获该错误
            except NameError:
                self.logger.debug('Field is Not Defined ' + field)
        yield weibo_item

    def parse(self, response):
        pass

'''
注意1：我们是点击“评论”进入详情页的，所以详情页前面师妹有超链接的
注意2：
'''